{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3c1d28eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/home/sczc619/run/LML/anaconda3/envs/tsne/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import os\n",
    "import random as rd\n",
    "import numpy as np\n",
    "import copy\n",
    "import copy as cp\n",
    "import dgl\n",
    "from collections import defaultdict\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import scipy.sparse as sp\n",
    "from scipy.io import loadmat\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "from torch.nn import TransformerEncoder, TransformerEncoderLayer\n",
    "from torch_geometric.utils import to_dense_adj, subgraph\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score, accuracy_score, recall_score, roc_auc_score, average_precision_score, confusion_matrix\n",
    "\n",
    "filelist = {\n",
    "    'amz_upu': 'amz_upu_adjlists.pickle',\n",
    "    'amz_usu': 'amz_usu_adjlists.pickle',\n",
    "    'amz_uvu': 'amz_uvu_adjlists.pickle',\n",
    "    'yelp_rsr': 'yelp_rsr_adjlists.pickle',\n",
    "    'yelp_rtr': 'yelp_rtr_adjlists.pickle',\n",
    "    'yelp_rur': 'yelp_rur_adjlists.pickle'\n",
    "}\n",
    "\n",
    "file_matrix_prefix = {\n",
    "    'amz_upu': 'amazon_upu_matrix_',\n",
    "    'amz_usu': 'amazon_usu_matrix_',\n",
    "    'amz_uvu': 'amazon_uvu_matrix_',\n",
    "    'yelp_rsr': 'yelpnet_rsr_matrix_decompision_',\n",
    "    'yelp_rtr': 'yelpnet_rtr_matrix_decompision_',\n",
    "    'yelp_rur': 'yelpnet_rur_matrix_decompision_'\n",
    "}\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def sample_batch_p(x, dist_row, batch_size, temperature=1.0):\n",
    "    \"\"\"\n",
    "        根据距离的概率分布采样邻居节点\n",
    "        :param x: 当前节点的索引\n",
    "        :param dist_row: 当前节点与其他所有节点的距离向量\n",
    "        :param batch_size: 采样批次的大小\n",
    "        :param temperature: 温度参数，控制采样的集中程度\n",
    "        :return: 采样得到的邻居节点索引列表\n",
    "    \"\"\"\n",
    "    probs = torch.softmax(-dist_row / temperature, dim=-1)\n",
    "    probs[x] = 0\n",
    "    probs = probs / probs.sum()\n",
    "    batch = torch.multinomial(probs, batch_size, replacement=False)\n",
    "    return batch.tolist()\n",
    "\n",
    "def sample_subgraph(batch_centers, dist_matrix, sample_size=20, temperature=0.1):\n",
    "    \"\"\"\n",
    "        采样子图，包含中心节点及其邻居节点\n",
    "        :param batch_centers: 中心节点的索引列表\n",
    "        :param dist_matrix: 节点间距离矩阵\n",
    "        :param sample_size: 每个中心节点采样的邻居数量\n",
    "        :param temperature: 温度参数\n",
    "        :return: 采样得到的子图节点索引列表\n",
    "    \"\"\"\n",
    "    sub_nodes = set(batch_centers)\n",
    "    for xi in batch_centers:\n",
    "        dist_row = dist_matrix[xi].cpu().numpy()\n",
    "        probs = np.exp(-dist_row / temperature)\n",
    "        probs[xi] = 0\n",
    "        probs /= probs.sum()\n",
    "        neighbors = np.random.choice(len(dist_row), size=sample_size, p=probs, replace=False)\n",
    "        sub_nodes.update(neighbors.tolist())\n",
    "    sub_nodes = list(sub_nodes)\n",
    "    return sub_nodes\n",
    "\n",
    "def get_subgraph_edges(edge_index, sub_nodes):\n",
    "    \"\"\"\n",
    "        提取子图包含的边\n",
    "        :param edge_index: 原图的边索引\n",
    "        :param sub_nodes: 子图的节点索引列表\n",
    "        :return: 子图的边索引\n",
    "    \"\"\"\n",
    "    mask = torch.isin(edge_index[0], sub_nodes) & torch.isin(edge_index[1], sub_nodes)\n",
    "    return edge_index[:, mask]\n",
    "\n",
    "def iterate_batches(indices, batch_size, shuffle=True):\n",
    "    \"\"\"\n",
    "        将索引列表划分为指定大小的批次\n",
    "        :param indices: 样本索引列表\n",
    "        :param batch_size: 每个批次的大小\n",
    "        :param shuffle: 是否打乱顺序\n",
    "        :return: 生成批次索引的迭代器\n",
    "    \"\"\"\n",
    "    if shuffle:\n",
    "        rd.shuffle(indices)\n",
    "    for i in range(0, len(indices), batch_size):\n",
    "        yield indices[i:i + batch_size]\n",
    "\n",
    "# def calculate_g_mean(y_true, y_pred):\n",
    "#     cm = confusion_matrix(y_true, y_pred)\n",
    "#     TP = cm[1, 1]\n",
    "#     TN = cm[0, 0]\n",
    "#     FP = cm[0, 1]\n",
    "#     FN = cm[1, 0]\n",
    "#     sensitivity = TP / (TP + FN)\n",
    "#     specificity = TN / (TN + FP)\n",
    "#     g_mean = np.sqrt(sensitivity * specificity)\n",
    "#     return g_mean\n",
    "\n",
    "def dict_to_edge_index(edge_dict):\n",
    "    source_nodes = []\n",
    "    target_nodes = []\n",
    "    for src, targets in edge_dict.items():\n",
    "        for target in targets:\n",
    "            source_nodes.append(src)\n",
    "            target_nodes.append(target)\n",
    "    edge_index = [source_nodes, target_nodes]\n",
    "    return torch.LongTensor(edge_index)\n",
    "\n",
    "def numpy_array_to_edge_index(np_array):\n",
    "    assert np_array.ndim == 2 and np_array.shape[0] == np_array.shape[1], \"Input must be a square matrix.\"\n",
    "    rows, cols = np.nonzero(np_array)\n",
    "    edge_index = np.vstack((rows, cols))\n",
    "    edge_index_tensor = torch.from_numpy(edge_index).long()\n",
    "    return edge_index_tensor\n",
    "\n",
    "def load_data(data, k=2, prefix=''):\n",
    "    pickle_file = {}\n",
    "    matrix_prefix = {}\n",
    "    for key in filelist:\n",
    "        pickle_file[key] = os.path.join(prefix, filelist[key])\n",
    "        matrix_prefix[key] = os.path.join(prefix, file_matrix_prefix[key])\n",
    "\n",
    "    if data == 'yelp':\n",
    "        data_file = loadmat(os.path.join(prefix, 'YelpChi.mat'))\n",
    "        labels = data_file['label'].flatten()\n",
    "        feat_data = data_file['features'].todense().A\n",
    "        with open(pickle_file['yelp_rur'], 'rb') as file:\n",
    "            relation1 = pickle.load(file)\n",
    "        file.close()\n",
    "        relation1 = dict_to_edge_index(relation1)\n",
    "        relation1_tree = []\n",
    "        for i in range(1, k + 1):\n",
    "            file_name = '{}{}.pkl'.format(matrix_prefix['yelp_rur'], i)\n",
    "            with open(file_name, 'rb') as file:\n",
    "                tree = pickle.load(file)\n",
    "            file.close()\n",
    "            relation1_tree.append(numpy_array_to_edge_index(tree))\n",
    "        with open(pickle_file['yelp_rtr'], 'rb') as file:\n",
    "            relation2 = pickle.load(file)\n",
    "        file.close()\n",
    "        relation2 = dict_to_edge_index(relation2)\n",
    "        relation2_tree = []\n",
    "        for i in range(1, k + 1):\n",
    "            file_name = '{}{}.pkl'.format(matrix_prefix['yelp_rtr'], i)\n",
    "            with open(file_name, 'rb') as file:\n",
    "                tree = pickle.load(file)\n",
    "            file.close()\n",
    "            relation2_tree.append(numpy_array_to_edge_index(tree))\n",
    "        with open(pickle_file['yelp_rsr'], 'rb') as file:\n",
    "            relation3 = pickle.load(file)\n",
    "        file.close()\n",
    "        relation3 = dict_to_edge_index(relation3)\n",
    "        relation3_tree = []\n",
    "        for i in range(1, k + 1):\n",
    "            file_name = '{}{}.pkl'.format(matrix_prefix['yelp_rsr'], i)\n",
    "            with open(file_name, 'rb') as file:\n",
    "                tree = pickle.load(file)\n",
    "            file.close()\n",
    "            relation3_tree.append(numpy_array_to_edge_index(tree))\n",
    "        return [[relation1, relation1_tree], [relation2, relation2_tree], [relation3, relation3_tree]], feat_data, labels\n",
    "    elif data == 'amazon':\n",
    "        data_file = loadmat(os.path.join(prefix, 'Amazon.mat'))\n",
    "        labels = data_file['label'].flatten()\n",
    "        feat_data = data_file['features'].todense().A\n",
    "        with open(pickle_file['amz_upu'], 'rb') as file:\n",
    "            relation1 = pickle.load(file)\n",
    "        file.close()\n",
    "        relation1 = dict_to_edge_index(relation1)\n",
    "        relation1_tree = []\n",
    "        for i in range(1, k + 1):\n",
    "            file_name = '{}{}.pkl'.format(matrix_prefix['amz_upu'], i)\n",
    "            with open(file_name, 'rb') as file:\n",
    "                tree = pickle.load(file)\n",
    "            file.close()\n",
    "            relation1_tree.append(numpy_array_to_edge_index(tree))\n",
    "        with open(pickle_file['amz_usu'], 'rb') as file:\n",
    "            relation2 = pickle.load(file)\n",
    "        file.close()\n",
    "        relation2 = dict_to_edge_index(relation2)\n",
    "        relation2_tree = []\n",
    "        for i in range(1, k + 1):\n",
    "            file_name = '{}{}.pkl'.format(matrix_prefix['amz_usu'], i)\n",
    "            with open(file_name, 'rb') as file:\n",
    "                tree = pickle.load(file)\n",
    "            file.close()\n",
    "            relation2_tree.append(numpy_array_to_edge_index(tree))\n",
    "        with open(pickle_file['amz_uvu'], 'rb') as file:\n",
    "            relation3 = pickle.load(file)\n",
    "        file.close()\n",
    "        relation3_tree = []\n",
    "        for i in range(1, k + 1):\n",
    "            file_name = '{}{}.pkl'.format(matrix_prefix['amz_uvu'], i)\n",
    "            with open(file_name, 'rb') as file:\n",
    "                tree = pickle.load(file)\n",
    "            file.close()\n",
    "            relation3_tree.append(numpy_array_to_edge_index(tree))\n",
    "        relation3 = dict_to_edge_index(relation3)\n",
    "\n",
    "        return [[relation1, relation1_tree], [relation2, relation2_tree], [relation3, relation3_tree]], feat_data, labels\n",
    "\n",
    "\n",
    "def Visualization(labels, embedding, prefix):\n",
    "    train_pos, train_neg = pos_neg_split(list(range(len(labels))), labels)\n",
    "    sampled_idx_train = undersample(train_pos, train_neg, scale=1)\n",
    "    tsne = TSNE(n_components=2, random_state=43)\n",
    "    sampled_idx_train = np.array(sampled_idx_train)\n",
    "    sampled_idx_train = np.random.choice(sampled_idx_train, size=5000, replace=True)\n",
    "    ps = embedding[sampled_idx_train]\n",
    "    ls = labels[sampled_idx_train]\n",
    "\n",
    "    X_reduced = tsne.fit_transform(ps)\n",
    "\n",
    "    scaler = MinMaxScaler(feature_range=(0, 1))\n",
    "    X_scaled = scaler.fit_transform(X_reduced)\n",
    "    print(X_scaled.shape)\n",
    "\n",
    "    plt.figure(figsize=(8, 8))\n",
    "\n",
    "    plt.scatter(X_scaled[ls == 0, 0], X_scaled[ls == 0, 1], c='#14517C', label='Label 0', s=3)\n",
    "\n",
    "    plt.scatter(X_scaled[ls == 1, 0], X_scaled[ls == 1, 1], c='#FA7F6F', label='Label 1', s=3)\n",
    "\n",
    "    ax = plt.gca()\n",
    "    ax.spines['top'].set_visible(False)\n",
    "    ax.spines['right'].set_visible(False)\n",
    "    ax.spines['left'].set_visible(False)\n",
    "    ax.spines['bottom'].set_visible(False)\n",
    "\n",
    "    plt.xticks([])\n",
    "    plt.yticks([])\n",
    "\n",
    "    plt.xlim(0, 1)\n",
    "    plt.ylim(0, 1)\n",
    "    filepath = os.path.join(prefix, 'HOGRL.png')\n",
    "    plt.savefig(filepath)\n",
    "    plt.show()\n",
    "\n",
    "def normalize(mx):\n",
    "    rowsum = np.array(mx.sum(1)) + 0.01\n",
    "    r_inv = np.power(rowsum, -1).flatten()\n",
    "    r_inv[np.isinf(r_inv)] = 0.\n",
    "    r_mat_inv = sp.diags(r_inv)\n",
    "    mx = r_mat_inv.dot(mx)\n",
    "    return mx\n",
    "\n",
    "def pos_neg_split(nodes, labels):\n",
    "    # 正负样本分割\n",
    "    pos_nodes = []\n",
    "    neg_nodes = cp.deepcopy(nodes)\n",
    "    aux_nodes = cp.deepcopy(nodes)\n",
    "    for idx, label in enumerate(labels):\n",
    "        if label == 1:\n",
    "            pos_nodes.append(aux_nodes[idx])\n",
    "            neg_nodes.remove(aux_nodes[idx])\n",
    "\n",
    "    return pos_nodes, neg_nodes\n",
    "\n",
    "def undersample(pos_nodes, neg_nodes, scale=1):\n",
    "    # 对负样本进行下采样，平衡正负样本数量\n",
    "    aux_nodes = cp.deepcopy(neg_nodes)\n",
    "    aux_nodes = rd.sample(aux_nodes, k=int(len(pos_nodes) * scale))\n",
    "    batch_nodes = pos_nodes + aux_nodes\n",
    "\n",
    "    return batch_nodes\n",
    "\n",
    "def calculate_g_mean(y_true, y_pred):\n",
    "    cm = confusion_matrix(y_true, y_pred)\n",
    "    sensitivities = []\n",
    "    for i in range(len(cm)):\n",
    "        TP = cm[i, i]\n",
    "        FN = cm[i, :].sum() - TP\n",
    "        sensitivity = TP / (TP + FN) if (TP + FN) != 0 else 0\n",
    "        sensitivities.append(sensitivity)\n",
    "    g_mean = np.prod(sensitivities) ** (1 / len(sensitivities))\n",
    "    return g_mean\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def test(idx_eval, y_eval, model, feat_data, edge_indexs, device):\n",
    "    model.eval()\n",
    "    # with torch.no_grad():\n",
    "    #     logits, _ = model(feat_data.to(device), edge_indexs, sub_nodes=None)\n",
    "    #     x_softmax = torch.exp(logits).cpu().detach()\n",
    "    #     positive_class_probs = x_softmax[:, 1].numpy()[np.array(idx_eval)]\n",
    "    #     auc_score = roc_auc_score(np.array(y_eval), np.array(positive_class_probs))\n",
    "    #     ap_score = average_precision_score(np.array(y_eval), np.array(positive_class_probs))\n",
    "    #     label_prob = (np.array(positive_class_probs) >= 0.5).astype(int)\n",
    "    #     f1_score_val = f1_score(np.array(y_eval), label_prob, average='macro')\n",
    "    #     g_mean = calculate_g_mean(np.array(y_eval), label_prob)\n",
    "    #\n",
    "    # return auc_score, ap_score, f1_score_val, g_mean\n",
    "    with torch.no_grad():\n",
    "        subgraph_data = []\n",
    "\n",
    "        for xi in idx_eval:\n",
    "            sub_nodes = [xi]\n",
    "            sub_edge_index = []\n",
    "            for rel_idx in range(len(edge_indexs)):\n",
    "                edge_index = edge_indexs[rel_idx][0].cpu().numpy()\n",
    "                mask = np.isin(edge_index[0], sub_nodes) & np.isin(edge_index[1], sub_nodes)\n",
    "                local_edges = edge_index[:, mask]\n",
    "\n",
    "                node_map = {n: i for i, n in enumerate(sub_nodes)}\n",
    "                src_nodes = [node_map[src] for src in local_edges[0]]\n",
    "                dst_nodes = [node_map[dst] for dst in local_edges[1]]\n",
    "\n",
    "                if len(src_nodes) > 0:\n",
    "                    edge_tensor = torch.tensor([src_nodes, dst_nodes], dtype=torch.long)\n",
    "                else:\n",
    "                    edge_tensor = torch.empty((2, 0), dtype=torch.long)\n",
    "\n",
    "                sub_edge_index.append(edge_tensor.to(device))\n",
    "\n",
    "            subgraph_data.append({\n",
    "                'features': feat_data[sub_nodes].clone().detach(),\n",
    "                'edges': sub_edge_index,\n",
    "                'global_idx': sub_nodes\n",
    "            })\n",
    "\n",
    "        logits, _ = model(subgraph_data)\n",
    "#         x_softmax = torch.exp(logits).cpu().detach()\n",
    "        x_softmax = torch.softmax(logits, dim=-1).cpu().numpy()\n",
    "        positive_class_probs = x_softmax[:, 0, 1]  # [batch_size, 1, 2] -> [64]\n",
    "        auc_score = roc_auc_score(np.array(y_eval), np.array(positive_class_probs))\n",
    "        ap_score = average_precision_score(np.array(y_eval), np.array(positive_class_probs))\n",
    "        label_prob = (np.array(positive_class_probs) >= 0.5).astype(int)\n",
    "        f1_score_val = f1_score(np.array(y_eval), label_prob, average='macro')\n",
    "        g_mean = calculate_g_mean(np.array(y_eval), label_prob)\n",
    "\n",
    "    return auc_score, ap_score, f1_score_val, g_mean\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "67a6990c",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class BSNE_Transformer(nn.Module):\n",
    "    def __init__(self, in_feat, out_feat, relation_nums=3, d_model=256,\n",
    "                 nhead=8, num_layers=3, dim_feedforward=256,\n",
    "                 drop_rate=0.5):\n",
    "        super().__init__()\n",
    "        self.relation_nums = relation_nums\n",
    "        self.d_model = d_model\n",
    "        self.nhead = nhead\n",
    "#         self.feature_proj = nn.Linear(in_feat, d_model)\n",
    "        self.feature_proj = nn.Sequential(\n",
    "            nn.Linear(in_feat, d_model),\n",
    "            nn.LayerNorm(d_model),\n",
    "            nn.ReLU() \n",
    "        )\n",
    "        self.norm = nn.LayerNorm(d_model) \n",
    "        self.relation_encoders = nn.ModuleList([\n",
    "            TransformerEncoder(\n",
    "                TransformerEncoderLayer(\n",
    "                    d_model=d_model,\n",
    "                    nhead=nhead,\n",
    "                    dim_feedforward=dim_feedforward,\n",
    "                    dropout=drop_rate,\n",
    "                    batch_first=True\n",
    "                ),\n",
    "                num_layers=num_layers\n",
    "            ) for _ in range(relation_nums)\n",
    "        ])\n",
    "\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Linear(relation_nums * d_model, 512),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(drop_rate),\n",
    "            nn.Linear(512, out_feat)\n",
    "        )\n",
    "\n",
    "    def forward(self, subgraph_batch):\n",
    "        max_nodes = max(sg['features'].size(0) for sg in subgraph_batch)\n",
    "        batch_size = len(subgraph_batch)\n",
    "        \n",
    "        # 特征填充和掩码生成\n",
    "        padded_features = []\n",
    "        padding_masks = {rel: [] for rel in range(self.relation_nums)}\n",
    "        \n",
    "        for sg in subgraph_batch:\n",
    "#             print(torch.isnan(sg['features']).sum())\n",
    "            num_real_nodes = sg['features'].size(0)\n",
    "            padded_feat = F.pad(sg['features'], (0, 0, 0, max_nodes - num_real_nodes))\n",
    "            padded_features.append(padded_feat)\n",
    "            \n",
    "            # 生成padding mask\n",
    "            pad_mask = torch.zeros(max_nodes, dtype=torch.bool, device=sg['features'].device)\n",
    "            pad_mask[num_real_nodes:] = True\n",
    "#             pad_mask[num_real_nodes:] = float('-inf')\n",
    "            for rel in range(self.relation_nums):\n",
    "                padding_masks[rel].append(pad_mask)\n",
    "                \n",
    "        \n",
    "        # 转换为批量张量\n",
    "        features = torch.stack(padded_features)  # [B, max_n, in_feat]\n",
    "#         print(\"Features before projection - mean:\", features.mean(), \"std:\", features.std(), \"nan:\", torch.isnan(features).any())\n",
    "        features = self.feature_proj(features)   # [B, max_n, d_model]\n",
    "#         print(\"features nan:\",torch.isnan(features).sum())\n",
    "#         print(\"Features after projection - mean:\", features.mean(), \"std:\", features.std(), \"nan:\", torch.isnan(features).any())\n",
    "        features = self.norm(features)\n",
    "\n",
    "        # 多关系处理\n",
    "        relation_outputs = []\n",
    "        for rel_idx in range(self.relation_nums):\n",
    "            adj_masks = []\n",
    "            for sg in subgraph_batch:\n",
    "                edge_index = sg['edges'][rel_idx]\n",
    "                adj = torch.zeros(max_nodes, max_nodes, dtype=torch.float, device=features.device)\n",
    "                if edge_index.size(1) > 0:\n",
    "                    src, dst = edge_index\n",
    "                    adj[src, dst] = 1.0 \n",
    "                adj_masks.append(adj)\n",
    "\n",
    "            # 创建邻接矩阵mask\n",
    "            adj_mask = torch.stack(adj_masks)  # [B, max_n, max_n]\n",
    "            adj_mask = adj_mask.masked_fill(adj_mask == 0.0, float('-inf'))\n",
    "            adj_mask = adj_mask.masked_fill(adj_mask == 1.0, 0.0)\n",
    "\n",
    "            # 处理padding mask\n",
    "            src_key_padding_mask = torch.stack(padding_masks[rel_idx])  # [B, max_n]\n",
    "\n",
    "            padding_3d_mask = src_key_padding_mask.unsqueeze(1) | src_key_padding_mask.unsqueeze(2)  # [B, max_n, max_n]\n",
    "\n",
    "            # 合并两种mask\n",
    "            combined_mask = adj_mask.masked_fill(padding_3d_mask, float('-inf'))\n",
    "            \n",
    "            combined_mask = combined_mask.unsqueeze(1)  # [B, 1, max_n, max_n]\n",
    "            combined_mask = combined_mask.expand(-1, self.nhead, -1, -1)  # [B, H, max_n, max_n]\n",
    "            combined_mask = combined_mask.reshape(-1, max_nodes, max_nodes)  # [B*H, max_n, max_n]\n",
    "            \n",
    "#             print(\"combined_mask nan:\",torch.isnan(combined_mask).sum())\n",
    "#             print('features:',features)\n",
    "#             print('adj_mask:',adj_mask)\n",
    "#             print('combined_mask:',combined_mask)\n",
    "            encoder_output = self.relation_encoders[rel_idx](\n",
    "                src=features,\n",
    "                mask=combined_mask, \n",
    "                src_key_padding_mask=None \n",
    "            )\n",
    "#             print(\"Input to transformer:\", features[0,:3,:3])\n",
    "            print(\"encoder_output nan:\",torch.isnan(encoder_output).sum())\n",
    "            relation_outputs.append(encoder_output)\n",
    "\n",
    "        # 合并多关系特征\n",
    "        combined = torch.cat(relation_outputs, dim=-1)  # [B, max_n, rel*d_model]\n",
    "        logits = self.classifier(combined)  # [B, max_n, out_feat]\n",
    "        logits = F.log_softmax(logits, dim=-1)\n",
    "        \n",
    "        tsne_feats = torch.stack(relation_outputs, dim=2).mean(dim=2)  # [B, max_n, d_model]\n",
    "        return logits, tsne_feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "178b76dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def calculate_tsne_loss(emb_p_list, emb_u_list, dist_matrix, batch_p_global_list, batch_u_global_list, sample_size=20, temperature=1, eps=1e-12):\n",
    "\n",
    "\n",
    "    total_loss = 0.0\n",
    "    batch_size = len(emb_p_list)\n",
    "    for emb_p, emb_u, sub_nodes, u_global_idx in zip(emb_p_list, emb_u_list, batch_p_global_list, batch_u_global_list):\n",
    "        # 采样\n",
    "        # 提取子图的全局节点索引\n",
    "        global_indices = sub_nodes\n",
    "        n_nodes = len(global_indices)\n",
    "\n",
    "        # 仅对中心节点进行采样\n",
    "        center_global_idx = sub_nodes[0]\n",
    "        center_local_idx = 0  # 在sub_nodes中的位置\n",
    "\n",
    "        # 获取中心节点到子图内其他节点的距离\n",
    "        dist_center = dist_matrix[center_global_idx, global_indices].cpu().numpy()\n",
    "        probs = np.exp(-dist_center)\n",
    "        probs[center_local_idx] = 0  # 排除自身\n",
    "        probs /= probs.sum()\n",
    "\n",
    "        # 采样邻居节点\n",
    "#         sampled_indices = np.random.choice(n_nodes, size=sample_size, p=probs, replace=False)\n",
    "        # 检查子图中的节点数是否小于 sample_size\n",
    "        if n_nodes - 1 < sample_size:  # 减去1是因为排除了中心节点自身\n",
    "            sample_size_actual = n_nodes - 1\n",
    "            if sample_size_actual <= 0:\n",
    "                sampled_indices = [center_local_idx]\n",
    "            else:\n",
    "                sampled_indices = np.random.choice(n_nodes, size=sample_size_actual, p=probs, replace=False)\n",
    "        else:\n",
    "            sampled_indices = np.random.choice(n_nodes, size=sample_size, p=probs, replace=False)\n",
    "        sampled_global = [global_indices[i] for i in sampled_indices]\n",
    "\n",
    "        # 提取采样后的嵌入和距离矩阵\n",
    "        emb_p_sampled = emb_p[sampled_indices]\n",
    "        dist_sampled = dist_matrix[sampled_global][:, sampled_global]\n",
    "\n",
    "        # --- Bu子图均匀采样 ---\n",
    "        u_sampled_indices = np.random.choice(emb_u.size(0), size=sample_size)\n",
    "        emb_u_sampled = emb_u[u_sampled_indices]\n",
    "        \n",
    "#         print(torch.isnan(emb_p_sampled).sum())\n",
    "#         print(torch.isnan(emb_u_sampled).sum())\n",
    "        # --- 计算局部损失 ---\n",
    "        P = (1.0 + dist_sampled ** 2) ** -1\n",
    "#         np.fill_diagonal(P, 0)\n",
    "        P.fill_diagonal_(0)\n",
    "        P = (P + P.T) / 2  # 确保对称\n",
    "        P = P / (P.sum(axis=1, keepdims=True) + eps)\n",
    "        P = torch.clamp(torch.tensor(P, device=emb_p.device), min=eps)\n",
    "\n",
    "        pairwise_dist = torch.cdist(emb_p_sampled, emb_p_sampled, p=2)\n",
    "        Q = (1.0 + pairwise_dist ** 2 / temperature) ** -1\n",
    "        Q.fill_diagonal_(0)\n",
    "        Q = (Q + Q.permute(1, 0)) / 2\n",
    "        Q = Q / (Q.sum(dim=1, keepdim=True) + eps)\n",
    "\n",
    "        loss_local = (P * (torch.log(P) - torch.log(Q))).sum()\n",
    "\n",
    "        # --- 全局损失计算 ---\n",
    "        sum_p_sub = P.sum(dim=1)\n",
    "        N = dist_matrix.shape[0]\n",
    "        k_Bp = (sum_p_sub * N / sample_size).mean()\n",
    "\n",
    "        d_bu = torch.exp(-torch.cdist(emb_p_sampled, emb_u_sampled, p=2))\n",
    "        sum_e_bu = d_bu.sum()\n",
    "        sum_e_bp = torch.exp(-pairwise_dist).sum()\n",
    "\n",
    "        global_ratio = k_Bp * (sum_e_bu / (sum_e_bp + eps))\n",
    "        loss_global = torch.log(global_ratio.clamp(min=eps))\n",
    "\n",
    "        total_loss += (loss_local + loss_global)\n",
    "\n",
    "    print(f\"local_loss: {loss_local.item()},global_loss: {loss_global.item()}\")\n",
    "\n",
    "    return total_loss / batch_size\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f7ebc07c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n",
      "loading data...\n",
      "\n",
      "=== Starting Pretraining ===\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[-0.7930, -0.7930, -0.2309,  ...,  0.1958, -0.7930,  1.2896],\n",
      "         [-0.1923, -0.6847, -0.6847,  ..., -0.3029, -0.6553,  2.5473],\n",
      "         [-0.8059, -0.8059, -0.2264,  ...,  0.2822, -0.8059,  1.3277],\n",
      "         ...,\n",
      "         [-0.6488, -0.6614, -0.6614,  ..., -0.6614, -0.6614,  1.8964],\n",
      "         [-0.7208, -0.7208, -0.7208,  ..., -0.7208, -0.7208,  1.8081],\n",
      "         [-0.7283, -0.7283, -0.7283,  ..., -0.7283, -0.7283,  2.2907]],\n",
      "\n",
      "        [[-0.5983, -0.6424, -0.6424,  ..., -0.5040, -0.6424,  2.0402],\n",
      "         [-0.5618, -0.6759, -0.6759,  ...,  0.2363, -0.6759,  1.9149],\n",
      "         [-0.4043, -0.6830, -0.6830,  ...,  0.1432, -0.6830,  2.2647],\n",
      "         ...,\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638],\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638],\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638]]],\n",
      "       device='cuda:0', grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(46080, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[-0.7930, -0.7930, -0.2309,  ...,  0.1958, -0.7930,  1.2896],\n",
      "         [-0.1923, -0.6847, -0.6847,  ..., -0.3029, -0.6553,  2.5473],\n",
      "         [-0.8059, -0.8059, -0.2264,  ...,  0.2822, -0.8059,  1.3277],\n",
      "         ...,\n",
      "         [-0.6488, -0.6614, -0.6614,  ..., -0.6614, -0.6614,  1.8964],\n",
      "         [-0.7208, -0.7208, -0.7208,  ..., -0.7208, -0.7208,  1.8081],\n",
      "         [-0.7283, -0.7283, -0.7283,  ..., -0.7283, -0.7283,  2.2907]],\n",
      "\n",
      "        [[-0.5983, -0.6424, -0.6424,  ..., -0.5040, -0.6424,  2.0402],\n",
      "         [-0.5618, -0.6759, -0.6759,  ...,  0.2363, -0.6759,  1.9149],\n",
      "         [-0.4043, -0.6830, -0.6830,  ...,  0.1432, -0.6830,  2.2647],\n",
      "         ...,\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638],\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638],\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638]]],\n",
      "       device='cuda:0', grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(46080, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[-0.7930, -0.7930, -0.2309,  ...,  0.1958, -0.7930,  1.2896],\n",
      "         [-0.1923, -0.6847, -0.6847,  ..., -0.3029, -0.6553,  2.5473],\n",
      "         [-0.8059, -0.8059, -0.2264,  ...,  0.2822, -0.8059,  1.3277],\n",
      "         ...,\n",
      "         [-0.6488, -0.6614, -0.6614,  ..., -0.6614, -0.6614,  1.8964],\n",
      "         [-0.7208, -0.7208, -0.7208,  ..., -0.7208, -0.7208,  1.8081],\n",
      "         [-0.7283, -0.7283, -0.7283,  ..., -0.7283, -0.7283,  2.2907]],\n",
      "\n",
      "        [[-0.5983, -0.6424, -0.6424,  ..., -0.5040, -0.6424,  2.0402],\n",
      "         [-0.5618, -0.6759, -0.6759,  ...,  0.2363, -0.6759,  1.9149],\n",
      "         [-0.4043, -0.6830, -0.6830,  ...,  0.1432, -0.6830,  2.2647],\n",
      "         ...,\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638],\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638],\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638]]],\n",
      "       device='cuda:0', grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(46080, device='cuda:0')\n",
      "embeddings_p nan : tensor(46080, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[-0.7618, -0.7618, -0.7618,  ..., -0.7618, -0.7618,  2.0181],\n",
      "         [-0.7967, -0.7080, -0.1810,  ...,  0.2759, -0.7967,  1.1239],\n",
      "         [-0.2559, -0.6549, -0.6549,  ..., -0.0273,  0.8982,  1.2555],\n",
      "         ...,\n",
      "         [-0.7957, -0.7273, -0.1989,  ...,  0.2507, -0.7957,  1.1545],\n",
      "         [-0.7970, -0.7440, -0.1797,  ...,  0.2578, -0.7970,  1.1752],\n",
      "         [-0.2799, -0.7730, -0.7186,  ...,  0.5964, -0.5883,  2.0537]],\n",
      "\n",
      "        [[-0.7355, -0.7355, -0.7355,  ..., -0.7355, -0.7355,  2.1848],\n",
      "         [-0.2759, -0.6798, -0.6798,  ..., -0.4137, -0.6798,  2.6846],\n",
      "         [-0.7155, -0.7155, -0.7155,  ..., -0.7155, -0.7155,  1.5829],\n",
      "         ...,\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638],\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638],\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638]]],\n",
      "       device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(46080, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[-0.7618, -0.7618, -0.7618,  ..., -0.7618, -0.7618,  2.0181],\n",
      "         [-0.7967, -0.7080, -0.1810,  ...,  0.2759, -0.7967,  1.1239],\n",
      "         [-0.2559, -0.6549, -0.6549,  ..., -0.0273,  0.8982,  1.2555],\n",
      "         ...,\n",
      "         [-0.7957, -0.7273, -0.1989,  ...,  0.2507, -0.7957,  1.1545],\n",
      "         [-0.7970, -0.7440, -0.1797,  ...,  0.2578, -0.7970,  1.1752],\n",
      "         [-0.2799, -0.7730, -0.7186,  ...,  0.5964, -0.5883,  2.0537]],\n",
      "\n",
      "        [[-0.7355, -0.7355, -0.7355,  ..., -0.7355, -0.7355,  2.1848],\n",
      "         [-0.2759, -0.6798, -0.6798,  ..., -0.4137, -0.6798,  2.6846],\n",
      "         [-0.7155, -0.7155, -0.7155,  ..., -0.7155, -0.7155,  1.5829],\n",
      "         ...,\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638],\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638],\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638]]],\n",
      "       device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., 0., -inf],\n",
      "         [0., 0., -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., 0., -inf],\n",
      "         [0., 0., -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., 0., -inf],\n",
      "         [0., 0., -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(46080, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[-0.7618, -0.7618, -0.7618,  ..., -0.7618, -0.7618,  2.0181],\n",
      "         [-0.7967, -0.7080, -0.1810,  ...,  0.2759, -0.7967,  1.1239],\n",
      "         [-0.2559, -0.6549, -0.6549,  ..., -0.0273,  0.8982,  1.2555],\n",
      "         ...,\n",
      "         [-0.7957, -0.7273, -0.1989,  ...,  0.2507, -0.7957,  1.1545],\n",
      "         [-0.7970, -0.7440, -0.1797,  ...,  0.2578, -0.7970,  1.1752],\n",
      "         [-0.2799, -0.7730, -0.7186,  ...,  0.5964, -0.5883,  2.0537]],\n",
      "\n",
      "        [[-0.7355, -0.7355, -0.7355,  ..., -0.7355, -0.7355,  2.1848],\n",
      "         [-0.2759, -0.6798, -0.6798,  ..., -0.4137, -0.6798,  2.6846],\n",
      "         [-0.7155, -0.7155, -0.7155,  ..., -0.7155, -0.7155,  1.5829],\n",
      "         ...,\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638],\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638],\n",
      "         [-0.7638, -0.7638, -0.7638,  ..., -0.7638, -0.7638, -0.7638]]],\n",
      "       device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(46080, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n",
      "Pretrain Epoch: 000, TSNE Loss: nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_5095/2046021845.py:51: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  P = torch.clamp(torch.tensor(P, device=emb_p.device), min=eps)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(103808, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(103808, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(103808, device='cuda:0')\n",
      "embeddings_p nan : tensor(103808, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(103808, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., 0.,  ..., 0., -inf, 0.],\n",
      "         ...,\n",
      "         [-inf, -inf, 0.,  ..., 0., -inf, 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, 0., 0.,  ..., 0., -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., 0.,  ..., 0., -inf, 0.],\n",
      "         ...,\n",
      "         [-inf, -inf, 0.,  ..., 0., -inf, 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, 0., 0.,  ..., 0., -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., 0.,  ..., 0., -inf, 0.],\n",
      "         ...,\n",
      "         [-inf, -inf, 0.,  ..., 0., -inf, 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, 0., 0.,  ..., 0., -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(103808, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(103808, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(39168, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(39168, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(39168, device='cuda:0')\n",
      "embeddings_p nan : tensor(39168, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(39168, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(39168, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(39168, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(50304, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(50304, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(50304, device='cuda:0')\n",
      "embeddings_p nan : tensor(50304, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(50304, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(50304, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(50304, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(256000, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(256000, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(256000, device='cuda:0')\n",
      "embeddings_p nan : tensor(256000, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(256000, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(256000, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(256000, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(127104, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(127104, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(127104, device='cuda:0')\n",
      "embeddings_p nan : tensor(127104, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(127104, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(127104, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(127104, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n",
      "Pretrain Epoch: 005, TSNE Loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(826240, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(826240, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(826240, device='cuda:0')\n",
      "embeddings_p nan : tensor(826240, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(826240, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(826240, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(826240, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(396928, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(396928, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(396928, device='cuda:0')\n",
      "embeddings_p nan : tensor(396928, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(396928, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(396928, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(396928, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(145536, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(145536, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(145536, device='cuda:0')\n",
      "embeddings_p nan : tensor(145536, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(145536, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(145536, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(145536, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(549760, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, 0.,  ..., 0., 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, 0.,  ..., 0., 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, 0.,  ..., 0., 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(549760, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(549760, device='cuda:0')\n",
      "embeddings_p nan : tensor(549760, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(549760, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(549760, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(549760, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(38784, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, 0.],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [0., -inf, 0.,  ..., 0., -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, 0.],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [0., -inf, 0.,  ..., 0., -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, 0.],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [0., -inf, 0.,  ..., 0., -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(38784, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(38784, device='cuda:0')\n",
      "embeddings_p nan : tensor(38784, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(38784, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, 0., -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, 0., -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, 0., -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(38784, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(38784, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n",
      "Pretrain Epoch: 010, TSNE Loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(271744, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(271744, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(271744, device='cuda:0')\n",
      "embeddings_p nan : tensor(271744, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(271744, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(271744, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(271744, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(466048, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(466048, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(466048, device='cuda:0')\n",
      "embeddings_p nan : tensor(466048, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(466048, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(466048, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(466048, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(318720, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(318720, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(318720, device='cuda:0')\n",
      "embeddings_p nan : tensor(318720, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(318720, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(318720, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(318720, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(67328, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., 0., 0., -inf],\n",
      "         ...,\n",
      "         [0., -inf, 0.,  ..., 0., 0., -inf],\n",
      "         [0., -inf, 0.,  ..., 0., 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., 0., 0., -inf],\n",
      "         ...,\n",
      "         [0., -inf, 0.,  ..., 0., 0., -inf],\n",
      "         [0., -inf, 0.,  ..., 0., 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., 0., 0., -inf],\n",
      "         ...,\n",
      "         [0., -inf, 0.,  ..., 0., 0., -inf],\n",
      "         [0., -inf, 0.,  ..., 0., 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(67328, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(67328, device='cuda:0')\n",
      "embeddings_p nan : tensor(67328, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(67328, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(67328, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(67328, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(15104, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(15104, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(15104, device='cuda:0')\n",
      "embeddings_p nan : tensor(15104, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(15104, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(15104, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(15104, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n",
      "Pretrain Epoch: 015, TSNE Loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(124160, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.],\n",
      "         [0., -inf, -inf,  ..., 0., 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(124160, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(124160, device='cuda:0')\n",
      "embeddings_p nan : tensor(124160, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(124160, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(124160, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(124160, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(38656, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [0., -inf, -inf,  ..., 0., 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(38656, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(38656, device='cuda:0')\n",
      "embeddings_p nan : tensor(38656, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(38656, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, 0., -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(38656, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(38656, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(57088, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(57088, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(57088, device='cuda:0')\n",
      "embeddings_p nan : tensor(57088, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(57088, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(57088, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(57088, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(162176, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.],\n",
      "         [0., -inf, -inf,  ..., -inf, 0., 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(162176, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [0., -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(162176, device='cuda:0')\n",
      "embeddings_p nan : tensor(162176, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(162176, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(162176, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, 0.],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [0., -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, -inf]]], device='cuda:0')\n",
      "encoder_output nan: tensor(162176, device='cuda:0')\n",
      "\n",
      "local_loss: nan,global_loss: nan\n",
      "Pretrain early stopping at epoch 19\n",
      "\n",
      "=== Starting Fine-tuning ===\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(5248, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., 0., 0.,  ..., -inf, 0., -inf],\n",
      "         [0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, 0., -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., 0., 0.,  ..., -inf, 0., -inf],\n",
      "         [0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, 0., -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., 0., 0.,  ..., -inf, 0., -inf],\n",
      "         [0., 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, 0., -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [0., -inf, 0.,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(5248, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         ...,\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan],\n",
      "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0',\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "adj_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]],\n",
      "\n",
      "        [[0., -inf, -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, 0., -inf,  ..., -inf, -inf, -inf],\n",
      "         [-inf, -inf, 0.,  ..., -inf, -inf, -inf],\n",
      "         ...,\n",
      "         [-inf, -inf, -inf,  ..., 0., -inf, -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, 0., -inf],\n",
      "         [-inf, -inf, -inf,  ..., -inf, -inf, 0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(5248, device='cuda:0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(110592, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(110592, device='cuda:0')\n",
      "combined_mask nan: tensor(0, device='cuda:0')\n",
      "features: tensor([[[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]],\n",
      "\n",
      "        [[nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')\n",
      "adj_mask: tensor([[[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]]], device='cuda:0')\n",
      "combined_mask: tensor([[[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]],\n",
      "\n",
      "        [[0.]]], device='cuda:0')\n",
      "encoder_output nan: tensor(110592, device='cuda:0')\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Input contains NaN.",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mValueError\u001b[39m                                Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[22]\u001b[39m\u001b[32m, line 305\u001b[39m\n\u001b[32m    302\u001b[39m writer.add_scalar(\u001b[33m'\u001b[39m\u001b[33mFineTune/Train_Loss\u001b[39m\u001b[33m'\u001b[39m, avg_loss, epoch)\n\u001b[32m    304\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m epoch % \u001b[32m5\u001b[39m == \u001b[32m0\u001b[39m:\n\u001b[32m--> \u001b[39m\u001b[32m305\u001b[39m     val_auc, val_ap, val_f1, val_g_mean = \u001b[43mtest\u001b[49m\u001b[43m(\u001b[49m\u001b[43midx_val\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_val\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbsne_model\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfeat_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43medge_indexs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    307\u001b[39m     writer.add_scalar(\u001b[33m'\u001b[39m\u001b[33mValidation/AUC\u001b[39m\u001b[33m'\u001b[39m, val_auc, epoch)\n\u001b[32m    308\u001b[39m     writer.add_scalar(\u001b[33m'\u001b[39m\u001b[33mValidation/F1\u001b[39m\u001b[33m'\u001b[39m, val_f1, epoch)\n",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 340\u001b[39m, in \u001b[36mtest\u001b[39m\u001b[34m(idx_eval, y_eval, model, feat_data, edge_indexs, device)\u001b[39m\n\u001b[32m    338\u001b[39m x_softmax = torch.softmax(logits, dim=-\u001b[32m1\u001b[39m).cpu().numpy()\n\u001b[32m    339\u001b[39m positive_class_probs = x_softmax[:, \u001b[32m0\u001b[39m, \u001b[32m1\u001b[39m]  \u001b[38;5;66;03m# [batch_size, 1, 2] -> [64]\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m340\u001b[39m auc_score = \u001b[43mroc_auc_score\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnp\u001b[49m\u001b[43m.\u001b[49m\u001b[43marray\u001b[49m\u001b[43m(\u001b[49m\u001b[43my_eval\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnp\u001b[49m\u001b[43m.\u001b[49m\u001b[43marray\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpositive_class_probs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    341\u001b[39m ap_score = average_precision_score(np.array(y_eval), np.array(positive_class_probs))\n\u001b[32m    342\u001b[39m label_prob = (np.array(positive_class_probs) >= \u001b[32m0.5\u001b[39m).astype(\u001b[38;5;28mint\u001b[39m)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/run/LML/anaconda3/envs/tsne/lib/python3.11/site-packages/sklearn/metrics/_ranking.py:551\u001b[39m, in \u001b[36mroc_auc_score\u001b[39m\u001b[34m(y_true, y_score, average, sample_weight, max_fpr, multi_class, labels)\u001b[39m\n\u001b[32m    549\u001b[39m y_type = type_of_target(y_true, input_name=\u001b[33m\"\u001b[39m\u001b[33my_true\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m    550\u001b[39m y_true = check_array(y_true, ensure_2d=\u001b[38;5;28;01mFalse\u001b[39;00m, dtype=\u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[32m--> \u001b[39m\u001b[32m551\u001b[39m y_score = \u001b[43mcheck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43my_score\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mensure_2d\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[32m    553\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m y_type == \u001b[33m\"\u001b[39m\u001b[33mmulticlass\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m (\n\u001b[32m    554\u001b[39m     y_type == \u001b[33m\"\u001b[39m\u001b[33mbinary\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m y_score.ndim == \u001b[32m2\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m y_score.shape[\u001b[32m1\u001b[39m] > \u001b[32m2\u001b[39m\n\u001b[32m    555\u001b[39m ):\n\u001b[32m    556\u001b[39m     \u001b[38;5;66;03m# do not support partial ROC computation for multiclass\u001b[39;00m\n\u001b[32m    557\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m max_fpr \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m max_fpr != \u001b[32m1.0\u001b[39m:\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/run/LML/anaconda3/envs/tsne/lib/python3.11/site-packages/sklearn/utils/validation.py:921\u001b[39m, in \u001b[36mcheck_array\u001b[39m\u001b[34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator, input_name)\u001b[39m\n\u001b[32m    915\u001b[39m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m    916\u001b[39m             \u001b[33m\"\u001b[39m\u001b[33mFound array with dim \u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[33m. \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m expected <= 2.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    917\u001b[39m             % (array.ndim, estimator_name)\n\u001b[32m    918\u001b[39m         )\n\u001b[32m    920\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m force_all_finite:\n\u001b[32m--> \u001b[39m\u001b[32m921\u001b[39m         \u001b[43m_assert_all_finite\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    922\u001b[39m \u001b[43m            \u001b[49m\u001b[43marray\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    923\u001b[39m \u001b[43m            \u001b[49m\u001b[43minput_name\u001b[49m\u001b[43m=\u001b[49m\u001b[43minput_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    924\u001b[39m \u001b[43m            \u001b[49m\u001b[43mestimator_name\u001b[49m\u001b[43m=\u001b[49m\u001b[43mestimator_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    925\u001b[39m \u001b[43m            \u001b[49m\u001b[43mallow_nan\u001b[49m\u001b[43m=\u001b[49m\u001b[43mforce_all_finite\u001b[49m\u001b[43m \u001b[49m\u001b[43m==\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mallow-nan\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m    926\u001b[39m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    928\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m ensure_min_samples > \u001b[32m0\u001b[39m:\n\u001b[32m    929\u001b[39m     n_samples = _num_samples(array)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/run/LML/anaconda3/envs/tsne/lib/python3.11/site-packages/sklearn/utils/validation.py:161\u001b[39m, in \u001b[36m_assert_all_finite\u001b[39m\u001b[34m(X, allow_nan, msg_dtype, estimator_name, input_name)\u001b[39m\n\u001b[32m    144\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m estimator_name \u001b[38;5;129;01mand\u001b[39;00m input_name == \u001b[33m\"\u001b[39m\u001b[33mX\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m has_nan_error:\n\u001b[32m    145\u001b[39m     \u001b[38;5;66;03m# Improve the error message on how to handle missing values in\u001b[39;00m\n\u001b[32m    146\u001b[39m     \u001b[38;5;66;03m# scikit-learn.\u001b[39;00m\n\u001b[32m    147\u001b[39m     msg_err += (\n\u001b[32m    148\u001b[39m         \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mestimator_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m does not accept missing values\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    149\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33m encoded as NaN natively. For supervised learning, you might want\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m   (...)\u001b[39m\u001b[32m    159\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33m#estimators-that-handle-nan-values\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    160\u001b[39m     )\n\u001b[32m--> \u001b[39m\u001b[32m161\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(msg_err)\n",
      "\u001b[31mValueError\u001b[39m: Input contains NaN."
     ]
    }
   ],
   "source": [
    "args = {\n",
    "    \"dataset\": \"amazon\",\n",
    "#     \"dataset\": \"yelp\",\n",
    "    \"batch_size\": 2,\n",
    "    \"sample_size\": 40,\n",
    "    \"weight_decay\": 0.00005,\n",
    "    \"emb_size\": 32,\n",
    "    \"pretrain_epochs\": 50,\n",
    "    \"pretrain_lr\": 0.001,\n",
    "    \"finetune_lr\": 0.0005,\n",
    "    \"num_epochs\": 100,\n",
    "    \"pretrain_patience\": 20,\n",
    "    \"patience\": 30,\n",
    "    \"tsne_weight\": 0.3,\n",
    "    \"weight\": 0.6,\n",
    "    \"layers\": 7,\n",
    "    \"test_size\": 0.6,\n",
    "    \"val_size\": 0.5,\n",
    "    \"layers_tree\": 7,\n",
    "    \"seed\": 76,\n",
    "    \"num_heads\": 2,\n",
    "    \"drop_rate\": 0.5\n",
    "}\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "# device = torch.device('cpu')\n",
    "print(device)\n",
    "\n",
    "timestamp = time.strftime(\"%Y%m%d-%H%M%S\")\n",
    "writer = SummaryWriter(f'runs/{args[\"dataset\"]}_{timestamp}')\n",
    "\n",
    "print('loading data...')\n",
    "prefix = \"../../data/\"\n",
    "\n",
    "edge_indexs, feat_data, labels = load_data(args['dataset'], args['layers_tree'], prefix)\n",
    "\n",
    "np.random.seed(args['seed'])\n",
    "rd.seed(args['seed'])\n",
    "\n",
    "if args['dataset'] == 'yelp':\n",
    "    index = list(range(len(labels)))\n",
    "    idx_train_val, idx_test, y_train_val, y_test = train_test_split(index, labels, stratify=labels,\n",
    "                                                                    test_size=args['test_size'], random_state=2,\n",
    "                                                                    shuffle=True)\n",
    "    idx_train, idx_val, y_train, y_val = train_test_split(idx_train_val, y_train_val,\n",
    "                                                          stratify=y_train_val, test_size=args['val_size'],\n",
    "                                                          random_state=2, shuffle=True)\n",
    "    dist_path = os.path.join(prefix, \"YelpChi_shortest_distance.pkl\")\n",
    "elif args['dataset'] == 'amazon':\n",
    "    index = list(range(3305, len(labels)))\n",
    "    idx_train_val, idx_test, y_train_val, y_test = train_test_split(index, labels[3305:],\n",
    "                                                                    stratify=labels[3305:],\n",
    "                                                                    test_size=args['test_size'],\n",
    "                                                                    random_state=2, shuffle=True)\n",
    "    idx_train, idx_val, y_train, y_val = train_test_split(idx_train_val, y_train_val,\n",
    "                                                          stratify=y_train_val, test_size=args['val_size'],\n",
    "                                                          random_state=2, shuffle=True)\n",
    "    dist_path = os.path.join(prefix, \"Amazon_shortest_distance.pkl\")\n",
    "\n",
    "with open(dist_path, 'rb') as f:\n",
    "    dist_data = pickle.load(f)\n",
    "    dist_matrix = torch.tensor(dist_data['dist_matrix']).to(device)\n",
    "\n",
    "\n",
    "adj_dict = defaultdict(list)\n",
    "for rel in edge_indexs:\n",
    "    edge_index = rel[0].cpu().numpy()\n",
    "    for src, dst in zip(edge_index[0], edge_index[1]):\n",
    "        adj_dict[src].append(dst)\n",
    "\n",
    "bsne_model = BSNE_Transformer(\n",
    "    in_feat=feat_data.shape[1],\n",
    "    out_feat=2,\n",
    "    relation_nums=len(edge_indexs),\n",
    "    d_model=64,\n",
    "    nhead=2,\n",
    "    num_layers=3,\n",
    "    dim_feedforward=256,\n",
    "    drop_rate=args['drop_rate']\n",
    "#     layers_tree=args['layers_tree'],\n",
    ").to(device)\n",
    "\n",
    "for edge_index in edge_indexs:\n",
    "    edge_index[0] = edge_index[0].to(device)\n",
    "    edge_index[1] = [tensor.to(device) for tensor in edge_index[1]]\n",
    "    \n",
    "# print(feat_data)\n",
    "feat_data = torch.tensor(feat_data).float().to(device)\n",
    "\n",
    "# feat_data = torch.tensor(feat_data).float()\n",
    "# feat_data = F.layer_norm(feat_data, [feat_data.size(1)])  # 层归一化\n",
    "# feat_data = feat_data.to(device)\n",
    "# print(feat_data)\n",
    "\n",
    "print(\"\\n=== Starting Pretraining ===\")\n",
    "\n",
    "bsne_model.classifier.requires_grad_(False)\n",
    "optimizer = torch.optim.AdamW(\n",
    "    filter(lambda p: p.requires_grad, bsne_model.parameters()),\n",
    "    lr=args['pretrain_lr'],\n",
    "    weight_decay=5e-5\n",
    ")\n",
    "pretrain_best_loss = float('inf')\n",
    "pretrain_no_improve = 0\n",
    "pretrain_early_stop = False\n",
    "\n",
    "# use_batchwise = args.get('use_batchwise', True)\n",
    "\n",
    "for epoch in range(args['pretrain_epochs']):\n",
    "    if pretrain_early_stop:\n",
    "        break\n",
    "\n",
    "    bsne_model.train()\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    # 采样中心节点\n",
    "    batch_centers = rd.sample(range(feat_data.shape[0]), args['batch_size'])\n",
    "    subgraphs = []\n",
    "    subgraph_data = []\n",
    "    batch_u_global_list = []\n",
    "\n",
    "    # 构建批量子图数据\n",
    "    for xi in batch_centers:\n",
    "        sub_nodes = []\n",
    "        dist_row = dist_matrix[xi].cpu().numpy()\n",
    "        neighbors = np.where(dist_row == 1)[0].tolist()\n",
    "        sub_nodes = [xi] + neighbors\n",
    "        subgraphs.append(sub_nodes)\n",
    "#         print(len(sub_nodes))\n",
    "        # 构建子图数据结构\n",
    "        sub_edge_index = []\n",
    "        for rel_idx in range(len(edge_indexs)):\n",
    "            edge_index = edge_indexs[rel_idx][0].cpu().numpy()\n",
    "            mask = np.isin(edge_index[0], sub_nodes) & np.isin(edge_index[1], sub_nodes)\n",
    "            local_edges = edge_index[:, mask]\n",
    "\n",
    "            node_map = {n: i for i, n in enumerate(sub_nodes)}\n",
    "            src_nodes = [node_map[src] for src in local_edges[0]] if len(local_edges[0]) > 0 else []\n",
    "            dst_nodes = [node_map[dst] for dst in local_edges[1]] if len(local_edges[1]) > 0 else []\n",
    "            \n",
    "            if len(src_nodes) > 0:\n",
    "                edge_tensor = torch.tensor([src_nodes, dst_nodes], dtype=torch.long)\n",
    "            else:\n",
    "                edge_tensor = torch.empty((2, 0), dtype=torch.long)\n",
    "\n",
    "            sub_edge_index.append(edge_tensor.to(device))\n",
    "\n",
    "        subgraph_data.append({\n",
    "            'features': feat_data[sub_nodes].clone().detach(),\n",
    "            'edges': sub_edge_index,\n",
    "            'global_idx': sub_nodes\n",
    "        })\n",
    "\n",
    "        bu_global = np.random.choice(feat_data.shape[0], size=len(sub_nodes), replace=False)\n",
    "        batch_u_global_list.append(bu_global)\n",
    "\n",
    "    _, embeddings_p = bsne_model(subgraph_data)\n",
    "    print(\"embeddings_p nan :\", torch.isnan(embeddings_p).sum())\n",
    "#     print(embeddings_p)\n",
    "#     print(torch.isnan(emb_u_sampled).sum())\n",
    "#     print(\"Features - mean:\", embeddings_p.mean(), \"std:\", embeddings_p.std(), \"nan:\", torch.isnan(embeddings_p).any())\n",
    "#     print(batch_u_global_list)\n",
    "    \n",
    "    subgraph_data_bu = []\n",
    "    for bu in batch_u_global_list:\n",
    "        feat_u = feat_data[bu]\n",
    "        # 为 Bu 子图构建 edges 属性\n",
    "        bu_edge_index = []\n",
    "        for rel_idx in range(len(edge_indexs)):\n",
    "            edge_index = edge_indexs[rel_idx][0].cpu().numpy()\n",
    "            mask = np.isin(edge_index[0], bu) & np.isin(edge_index[1], bu)\n",
    "            local_edges = edge_index[:, mask]\n",
    "\n",
    "            node_map = {n: i for i, n in enumerate(bu)}\n",
    "            src_nodes = [node_map[src] for src in local_edges[0]] if len(local_edges[0]) > 0 else []\n",
    "            dst_nodes = [node_map[dst] for dst in local_edges[1]] if len(local_edges[1]) > 0 else []\n",
    "\n",
    "#             if len(src_nodes) > 0:\n",
    "#                 edge_tensor = torch.tensor([src_nodes, dst_nodes], dtype=torch.long)\n",
    "#             else:\n",
    "#                 edge_tensor = torch.empty((2, 0), dtype=torch.long)\n",
    "            edge_tensor = torch.tensor([src_nodes, dst_nodes], dtype=torch.long)\n",
    "\n",
    "            bu_edge_index.append(edge_tensor.to(device))\n",
    "\n",
    "        subgraph_data_bu.append({\n",
    "            'features': feat_u,\n",
    "            'edges': bu_edge_index,\n",
    "            'global_idx': bu\n",
    "        })\n",
    "\n",
    "    with torch.no_grad():\n",
    "        _, embeddings_u = bsne_model(subgraph_data_bu)\n",
    "    \n",
    "    embeddings_p = embeddings_p.to(device)\n",
    "    embeddings_u = embeddings_u.to(device)\n",
    "    print()\n",
    "    tsne_loss = calculate_tsne_loss(\n",
    "        embeddings_p,\n",
    "        embeddings_u,\n",
    "        dist_matrix,\n",
    "        batch_p_global_list=[sg['global_idx'] for sg in subgraph_data],\n",
    "        batch_u_global_list=batch_u_global_list,\n",
    "        sample_size=args['sample_size'],\n",
    "        temperature=100,\n",
    "        eps=1e-10\n",
    "    ) * args['tsne_weight']\n",
    "\n",
    "    tsne_loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    if tsne_loss.item() < pretrain_best_loss:\n",
    "        pretrain_best_loss = tsne_loss.item()\n",
    "        pretrain_no_improve = 0\n",
    "    else:\n",
    "        pretrain_no_improve += 1\n",
    "\n",
    "    if pretrain_no_improve >= args['pretrain_patience']:\n",
    "        print(f\"Pretrain early stopping at epoch {epoch}\")\n",
    "        pretrain_early_stop = True\n",
    "\n",
    "    writer.add_scalar('Pretrain/TSNE_Loss', tsne_loss.item(), epoch)\n",
    "\n",
    "    if epoch % 5 == 0:\n",
    "        print(f'Pretrain Epoch: {epoch:03d}, TSNE Loss: {tsne_loss.item():.4f}')\n",
    "\n",
    "print(\"\\n=== Starting Fine-tuning ===\")\n",
    "bsne_model.classifier.requires_grad_(True)\n",
    "optimizer = torch.optim.AdamW(\n",
    "    filter(lambda p: p.requires_grad, bsne_model.parameters()),\n",
    "    lr=args['finetune_lr'],\n",
    "    weight_decay=5e-5\n",
    ")\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n",
    "    optimizer, mode='max', factor=0.5, patience=10, verbose=True\n",
    ")\n",
    "\n",
    "best_val_auc = 0.0\n",
    "best_model_state = None\n",
    "train_pos, train_neg = pos_neg_split(idx_train, y_train)\n",
    "\n",
    "no_improve_epochs = 0\n",
    "early_stop = False\n",
    "\n",
    "for epoch in range(args['num_epochs']):\n",
    "    if early_stop:\n",
    "        break\n",
    "\n",
    "    bsne_model.train()\n",
    "    total_loss = 0.0\n",
    "\n",
    "    batch_centers = rd.sample(train_pos + train_neg, args['batch_size'])\n",
    "    subgraphs = []\n",
    "    subgraph_data = []\n",
    "\n",
    "    for xi in batch_centers:\n",
    "        dist_row = dist_matrix[xi].cpu().numpy()\n",
    "        probs = np.exp(-dist_row)\n",
    "        probs[xi] = 0\n",
    "        probs /= probs.sum()\n",
    "        neighbors = np.random.choice(len(dist_row), size=args['sample_size'], p=probs, replace=False)\n",
    "        sub_nodes = [xi] + neighbors.tolist()\n",
    "\n",
    "        sub_edge_index = []\n",
    "        for rel_idx in range(len(edge_indexs)):\n",
    "            edge_index = edge_indexs[rel_idx][0].cpu().numpy()\n",
    "            mask = np.isin(edge_index[0], sub_nodes) & np.isin(edge_index[1], sub_nodes)\n",
    "            local_edges = edge_index[:, mask]\n",
    "\n",
    "            node_map = {n: i for i, n in enumerate(sub_nodes)}\n",
    "            src_nodes = [node_map[src] for src in local_edges[0]]\n",
    "            dst_nodes = [node_map[dst] for dst in local_edges[1]]\n",
    "\n",
    "            if len(src_nodes) > 0:\n",
    "                edge_tensor = torch.tensor([src_nodes, dst_nodes], dtype=torch.long)\n",
    "            else:\n",
    "                edge_tensor = torch.empty((2, 0), dtype=torch.long)\n",
    "\n",
    "            sub_edge_index.append(edge_tensor.to(device))\n",
    "\n",
    "        subgraph_data.append({\n",
    "            'features': feat_data[sub_nodes].clone().detach(),\n",
    "            'edges': sub_edge_index,\n",
    "            'global_idx': sub_nodes\n",
    "        })\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    out, _ = bsne_model(subgraph_data)\n",
    "#     labels_sub = torch.cat([torch.tensor(labels[subgraph['global_idx']]) for subgraph in subgraph_data]).to(device)\n",
    "    labels_sub = torch.tensor([labels[subgraph['global_idx'][0]] for subgraph in subgraph_data]).to(device)\n",
    "    out_centers = out[:, 0, : ].float()\n",
    "    labels_sub = labels_sub.long()\n",
    "    cls_loss = F.nll_loss(out_centers, labels_sub)\n",
    "\n",
    "    cls_loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    total_loss += cls_loss.item()\n",
    "\n",
    "    avg_loss = total_loss / args['batch_size']\n",
    "    writer.add_scalar('FineTune/Train_Loss', avg_loss, epoch)\n",
    "\n",
    "    if epoch % 5 == 0:\n",
    "        val_auc, val_ap, val_f1, val_g_mean = test(idx_val, y_val, bsne_model, feat_data, edge_indexs, device)\n",
    "\n",
    "        writer.add_scalar('Validation/AUC', val_auc, epoch)\n",
    "        writer.add_scalar('Validation/F1', val_f1, epoch)\n",
    "        writer.add_scalar('Validation/GMean', val_g_mean, epoch)\n",
    "\n",
    "        print(f'Epoch: {epoch:03d} | Loss: {avg_loss:.4f} | Val AUC: {val_auc:.4f} | Val F1: {val_f1:.4f}')\n",
    "\n",
    "        scheduler.step(val_auc)\n",
    "\n",
    "        if val_auc > best_val_auc:\n",
    "            best_val_auc = val_auc\n",
    "            no_improve_epochs = 0\n",
    "            best_model_state = copy.deepcopy(bsne_model.state_dict())\n",
    "        else:\n",
    "            no_improve_epochs += 1\n",
    "\n",
    "        if no_improve_epochs >= args['patience']:\n",
    "            print(f\"Early stopping at epoch {epoch}\")\n",
    "            early_stop = True\n",
    "\n",
    "bsne_model.load_state_dict(best_model_state)\n",
    "test_auc, test_ap, test_f1, test_g_mean = test(idx_test, y_test, bsne_model, feat_data, edge_indexs, device)\n",
    "print(f'\\n=== Final Test Results ===')\n",
    "print(f'Test AUC: {test_auc:.4f} | Test AP: {test_ap:.4f} | Test F1: {test_f1:.4f} | G-mean: {test_g_mean:.4f}')\n",
    "writer.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "9551d4b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'features': tensor([[ 2.0000, 43.0000,  0.0000,  ...,  0.0000, 18.5000,  1.0000],\n",
       "         [ 1.0000, 12.0000,  0.0000,  ...,  1.0000, 11.0000,  1.0000],\n",
       "         [ 1.0000,  5.0000,  0.0000,  ...,  1.0000, 12.0000,  1.0000],\n",
       "         ...,\n",
       "         [ 1.0000,  8.0000,  0.0000,  ...,  1.0000, 48.0000,  1.0000],\n",
       "         [ 1.0000, 14.0000,  1.0000,  ...,  1.0000, 28.0000, -1.0000],\n",
       "         [ 1.0000,  9.0000,  0.0000,  ...,  1.0000, 28.0000,  1.0000]],\n",
       "        device='cuda:0'),\n",
       " 'edges': [tensor([[ 381,  381,  381,  ..., 1510, 1512, 1525],\n",
       "          [ 679,  224,  647,  ..., 1510, 1512, 1525]], device='cuda:0'),\n",
       "  tensor([[ 95,  95,  95,  ..., 183, 183, 183],\n",
       "          [467, 739, 630,  ..., 199, 201, 257]], device='cuda:0'),\n",
       "  tensor([[ 437,  437,  437,  ..., 1243, 1259, 1054],\n",
       "          [   1, 1089,  760,  ..., 1243, 1259, 1054]], device='cuda:0')],\n",
       " 'global_idx': [4614,\n",
       "  34,\n",
       "  40,\n",
       "  50,\n",
       "  88,\n",
       "  96,\n",
       "  113,\n",
       "  121,\n",
       "  171,\n",
       "  180,\n",
       "  221,\n",
       "  265,\n",
       "  266,\n",
       "  290,\n",
       "  312,\n",
       "  352,\n",
       "  386,\n",
       "  458,\n",
       "  467,\n",
       "  494,\n",
       "  501,\n",
       "  519,\n",
       "  548,\n",
       "  565,\n",
       "  567,\n",
       "  575,\n",
       "  664,\n",
       "  694,\n",
       "  707,\n",
       "  717,\n",
       "  721,\n",
       "  750,\n",
       "  783,\n",
       "  784,\n",
       "  785,\n",
       "  855,\n",
       "  863,\n",
       "  873,\n",
       "  889,\n",
       "  891,\n",
       "  904,\n",
       "  935,\n",
       "  937,\n",
       "  958,\n",
       "  964,\n",
       "  970,\n",
       "  983,\n",
       "  984,\n",
       "  987,\n",
       "  990,\n",
       "  992,\n",
       "  1013,\n",
       "  1026,\n",
       "  1050,\n",
       "  1077,\n",
       "  1115,\n",
       "  1118,\n",
       "  1126,\n",
       "  1134,\n",
       "  1145,\n",
       "  1149,\n",
       "  1160,\n",
       "  1199,\n",
       "  1228,\n",
       "  1283,\n",
       "  1327,\n",
       "  1359,\n",
       "  1371,\n",
       "  1372,\n",
       "  1382,\n",
       "  1383,\n",
       "  1413,\n",
       "  1425,\n",
       "  1432,\n",
       "  1461,\n",
       "  1527,\n",
       "  1534,\n",
       "  1562,\n",
       "  1566,\n",
       "  1604,\n",
       "  1653,\n",
       "  1673,\n",
       "  1702,\n",
       "  1717,\n",
       "  1730,\n",
       "  1736,\n",
       "  1806,\n",
       "  1820,\n",
       "  1845,\n",
       "  1853,\n",
       "  1860,\n",
       "  1879,\n",
       "  1939,\n",
       "  1983,\n",
       "  2009,\n",
       "  2019,\n",
       "  2024,\n",
       "  2025,\n",
       "  2040,\n",
       "  2042,\n",
       "  2055,\n",
       "  2075,\n",
       "  2079,\n",
       "  2081,\n",
       "  2111,\n",
       "  2156,\n",
       "  2185,\n",
       "  2212,\n",
       "  2221,\n",
       "  2232,\n",
       "  2250,\n",
       "  2259,\n",
       "  2273,\n",
       "  2287,\n",
       "  2363,\n",
       "  2378,\n",
       "  2388,\n",
       "  2398,\n",
       "  2400,\n",
       "  2460,\n",
       "  2464,\n",
       "  2465,\n",
       "  2478,\n",
       "  2485,\n",
       "  2501,\n",
       "  2509,\n",
       "  2539,\n",
       "  2632,\n",
       "  2635,\n",
       "  2638,\n",
       "  2640,\n",
       "  2641,\n",
       "  2670,\n",
       "  2747,\n",
       "  2764,\n",
       "  2770,\n",
       "  2805,\n",
       "  2811,\n",
       "  2854,\n",
       "  2869,\n",
       "  2879,\n",
       "  2901,\n",
       "  2919,\n",
       "  2930,\n",
       "  2965,\n",
       "  2980,\n",
       "  2981,\n",
       "  2988,\n",
       "  3025,\n",
       "  3042,\n",
       "  3067,\n",
       "  3078,\n",
       "  3088,\n",
       "  3092,\n",
       "  3098,\n",
       "  3101,\n",
       "  3103,\n",
       "  3176,\n",
       "  3190,\n",
       "  3203,\n",
       "  3211,\n",
       "  3226,\n",
       "  3234,\n",
       "  3265,\n",
       "  3277,\n",
       "  3279,\n",
       "  3312,\n",
       "  3315,\n",
       "  3327,\n",
       "  3330,\n",
       "  3532,\n",
       "  3712,\n",
       "  3739,\n",
       "  3750,\n",
       "  3754,\n",
       "  3760,\n",
       "  3763,\n",
       "  3767,\n",
       "  3771,\n",
       "  3779,\n",
       "  3785,\n",
       "  3808,\n",
       "  3811,\n",
       "  3821,\n",
       "  3828,\n",
       "  3835,\n",
       "  3837,\n",
       "  3843,\n",
       "  3847,\n",
       "  3852,\n",
       "  3854,\n",
       "  3864,\n",
       "  3874,\n",
       "  3875,\n",
       "  3877,\n",
       "  3900,\n",
       "  3902,\n",
       "  3905,\n",
       "  3923,\n",
       "  3924,\n",
       "  3928,\n",
       "  3938,\n",
       "  3940,\n",
       "  3966,\n",
       "  3968,\n",
       "  4006,\n",
       "  4008,\n",
       "  4009,\n",
       "  4025,\n",
       "  4026,\n",
       "  4072,\n",
       "  4076,\n",
       "  4078,\n",
       "  4080,\n",
       "  4081,\n",
       "  4083,\n",
       "  4084,\n",
       "  4085,\n",
       "  4087,\n",
       "  4089,\n",
       "  4090,\n",
       "  4091,\n",
       "  4095,\n",
       "  4102,\n",
       "  4106,\n",
       "  4146,\n",
       "  4148,\n",
       "  4151,\n",
       "  4156,\n",
       "  4174,\n",
       "  4189,\n",
       "  4191,\n",
       "  4214,\n",
       "  4221,\n",
       "  4223,\n",
       "  4230,\n",
       "  4231,\n",
       "  4232,\n",
       "  4233,\n",
       "  4279,\n",
       "  4280,\n",
       "  4281,\n",
       "  4283,\n",
       "  4284,\n",
       "  4305,\n",
       "  4317,\n",
       "  4335,\n",
       "  4354,\n",
       "  4373,\n",
       "  4379,\n",
       "  4400,\n",
       "  4410,\n",
       "  4419,\n",
       "  4423,\n",
       "  4424,\n",
       "  4425,\n",
       "  4447,\n",
       "  4454,\n",
       "  4464,\n",
       "  4471,\n",
       "  4473,\n",
       "  4487,\n",
       "  4504,\n",
       "  4506,\n",
       "  4507,\n",
       "  4509,\n",
       "  4510,\n",
       "  4512,\n",
       "  4514,\n",
       "  4517,\n",
       "  4518,\n",
       "  4519,\n",
       "  4520,\n",
       "  4521,\n",
       "  4522,\n",
       "  4523,\n",
       "  4525,\n",
       "  4527,\n",
       "  4528,\n",
       "  4531,\n",
       "  4533,\n",
       "  4534,\n",
       "  4535,\n",
       "  4536,\n",
       "  4538,\n",
       "  4541,\n",
       "  4543,\n",
       "  4544,\n",
       "  4546,\n",
       "  4547,\n",
       "  4548,\n",
       "  4549,\n",
       "  4550,\n",
       "  4551,\n",
       "  4552,\n",
       "  4553,\n",
       "  4554,\n",
       "  4555,\n",
       "  4556,\n",
       "  4558,\n",
       "  4559,\n",
       "  4560,\n",
       "  4561,\n",
       "  4564,\n",
       "  4565,\n",
       "  4567,\n",
       "  4568,\n",
       "  4569,\n",
       "  4571,\n",
       "  4572,\n",
       "  4576,\n",
       "  4589,\n",
       "  4591,\n",
       "  4606,\n",
       "  4607,\n",
       "  4611,\n",
       "  4612,\n",
       "  4613,\n",
       "  4615,\n",
       "  4616,\n",
       "  4617,\n",
       "  4618,\n",
       "  4619,\n",
       "  4620,\n",
       "  4621,\n",
       "  4625,\n",
       "  4626,\n",
       "  4627,\n",
       "  4628,\n",
       "  4630,\n",
       "  4631,\n",
       "  4632,\n",
       "  4634,\n",
       "  4635,\n",
       "  4637,\n",
       "  4641,\n",
       "  4642,\n",
       "  4644,\n",
       "  4647,\n",
       "  4649,\n",
       "  4653,\n",
       "  4658,\n",
       "  4662,\n",
       "  4663,\n",
       "  4664,\n",
       "  4667,\n",
       "  4670,\n",
       "  4676,\n",
       "  4677,\n",
       "  4678,\n",
       "  4689,\n",
       "  4690,\n",
       "  4694,\n",
       "  4695,\n",
       "  4696,\n",
       "  4697,\n",
       "  4703,\n",
       "  4705,\n",
       "  4711,\n",
       "  4712,\n",
       "  4716,\n",
       "  4719,\n",
       "  4729,\n",
       "  4736,\n",
       "  4737,\n",
       "  4748,\n",
       "  4749,\n",
       "  4753,\n",
       "  4755,\n",
       "  4764,\n",
       "  4769,\n",
       "  4774,\n",
       "  4806,\n",
       "  4809,\n",
       "  4811,\n",
       "  4822,\n",
       "  4825,\n",
       "  4829,\n",
       "  4830,\n",
       "  4831,\n",
       "  4849,\n",
       "  4850,\n",
       "  4851,\n",
       "  4855,\n",
       "  4864,\n",
       "  4870,\n",
       "  4871,\n",
       "  4872,\n",
       "  4873,\n",
       "  4875,\n",
       "  4876,\n",
       "  4877,\n",
       "  4880,\n",
       "  4885,\n",
       "  4886,\n",
       "  4907,\n",
       "  4918,\n",
       "  4919,\n",
       "  4920,\n",
       "  4922,\n",
       "  4931,\n",
       "  4936,\n",
       "  4937,\n",
       "  4938,\n",
       "  4939,\n",
       "  4941,\n",
       "  4942,\n",
       "  4943,\n",
       "  4944,\n",
       "  4945,\n",
       "  4946,\n",
       "  4947,\n",
       "  4950,\n",
       "  4951,\n",
       "  4952,\n",
       "  4953,\n",
       "  4955,\n",
       "  4956,\n",
       "  4957,\n",
       "  4959,\n",
       "  4960,\n",
       "  4962,\n",
       "  4963,\n",
       "  4964,\n",
       "  4965,\n",
       "  4966,\n",
       "  4971,\n",
       "  4973,\n",
       "  4986,\n",
       "  4993,\n",
       "  4996,\n",
       "  4999,\n",
       "  5000,\n",
       "  5002,\n",
       "  5003,\n",
       "  5005,\n",
       "  5007,\n",
       "  5009,\n",
       "  5011,\n",
       "  5016,\n",
       "  5018,\n",
       "  5020,\n",
       "  5026,\n",
       "  5032,\n",
       "  5033,\n",
       "  5034,\n",
       "  5042,\n",
       "  5043,\n",
       "  5046,\n",
       "  5047,\n",
       "  5072,\n",
       "  5086,\n",
       "  5087,\n",
       "  5089,\n",
       "  5090,\n",
       "  5093,\n",
       "  5095,\n",
       "  5097,\n",
       "  5104,\n",
       "  5106,\n",
       "  5112,\n",
       "  5115,\n",
       "  5118,\n",
       "  5119,\n",
       "  5120,\n",
       "  5122,\n",
       "  5125,\n",
       "  5134,\n",
       "  5144,\n",
       "  5148,\n",
       "  5149,\n",
       "  5163,\n",
       "  5167,\n",
       "  5168,\n",
       "  5170,\n",
       "  5173,\n",
       "  5174,\n",
       "  5177,\n",
       "  5179,\n",
       "  5187,\n",
       "  5189,\n",
       "  5190,\n",
       "  5191,\n",
       "  5192,\n",
       "  5195,\n",
       "  5200,\n",
       "  5218,\n",
       "  5219,\n",
       "  5220,\n",
       "  5221,\n",
       "  5222,\n",
       "  5224,\n",
       "  5225,\n",
       "  5226,\n",
       "  5227,\n",
       "  5228,\n",
       "  5229,\n",
       "  5232,\n",
       "  5237,\n",
       "  5238,\n",
       "  5239,\n",
       "  5245,\n",
       "  5250,\n",
       "  5252,\n",
       "  5254,\n",
       "  5256,\n",
       "  5258,\n",
       "  5259,\n",
       "  5266,\n",
       "  5269,\n",
       "  5270,\n",
       "  5280,\n",
       "  5281,\n",
       "  5287,\n",
       "  5290,\n",
       "  5292,\n",
       "  5295,\n",
       "  5301,\n",
       "  5306,\n",
       "  5309,\n",
       "  5312,\n",
       "  5317,\n",
       "  5325,\n",
       "  5326,\n",
       "  5328,\n",
       "  5331,\n",
       "  5335,\n",
       "  5347,\n",
       "  5349,\n",
       "  5357,\n",
       "  5369,\n",
       "  5372,\n",
       "  5374,\n",
       "  5376,\n",
       "  5380,\n",
       "  5381,\n",
       "  5384,\n",
       "  5390,\n",
       "  5393,\n",
       "  5396,\n",
       "  5399,\n",
       "  5400,\n",
       "  5401,\n",
       "  5402,\n",
       "  5404,\n",
       "  5406,\n",
       "  5413,\n",
       "  5417,\n",
       "  5418,\n",
       "  5420,\n",
       "  5426,\n",
       "  5427,\n",
       "  5429,\n",
       "  5431,\n",
       "  5438,\n",
       "  5453,\n",
       "  5457,\n",
       "  5458,\n",
       "  5464,\n",
       "  5466,\n",
       "  5471,\n",
       "  5476,\n",
       "  5477,\n",
       "  5478,\n",
       "  5479,\n",
       "  5480,\n",
       "  5483,\n",
       "  5485,\n",
       "  5486,\n",
       "  5488,\n",
       "  5491,\n",
       "  5496,\n",
       "  5497,\n",
       "  5500,\n",
       "  5501,\n",
       "  5502,\n",
       "  5506,\n",
       "  5512,\n",
       "  5518,\n",
       "  5519,\n",
       "  5524,\n",
       "  5529,\n",
       "  5533,\n",
       "  5535,\n",
       "  5545,\n",
       "  5554,\n",
       "  5557,\n",
       "  5561,\n",
       "  5562,\n",
       "  5573,\n",
       "  5581,\n",
       "  5588,\n",
       "  5589,\n",
       "  5606,\n",
       "  5607,\n",
       "  5610,\n",
       "  5614,\n",
       "  5617,\n",
       "  5618,\n",
       "  5619,\n",
       "  5624,\n",
       "  5626,\n",
       "  5627,\n",
       "  5628,\n",
       "  5629,\n",
       "  5635,\n",
       "  5636,\n",
       "  5641,\n",
       "  5643,\n",
       "  5644,\n",
       "  5646,\n",
       "  5647,\n",
       "  5648,\n",
       "  5649,\n",
       "  5650,\n",
       "  5654,\n",
       "  5655,\n",
       "  5656,\n",
       "  5658,\n",
       "  5659,\n",
       "  5660,\n",
       "  5662,\n",
       "  5664,\n",
       "  5673,\n",
       "  5678,\n",
       "  5683,\n",
       "  5687,\n",
       "  5700,\n",
       "  5710,\n",
       "  5712,\n",
       "  5714,\n",
       "  5715,\n",
       "  5723,\n",
       "  5725,\n",
       "  5728,\n",
       "  5729,\n",
       "  5731,\n",
       "  5734,\n",
       "  5737,\n",
       "  5744,\n",
       "  5746,\n",
       "  5750,\n",
       "  5751,\n",
       "  5760,\n",
       "  5761,\n",
       "  5762,\n",
       "  5765,\n",
       "  5771,\n",
       "  5774,\n",
       "  5776,\n",
       "  5777,\n",
       "  5778,\n",
       "  5781,\n",
       "  5782,\n",
       "  5784,\n",
       "  5786,\n",
       "  5795,\n",
       "  5799,\n",
       "  5802,\n",
       "  5804,\n",
       "  5805,\n",
       "  5809,\n",
       "  5811,\n",
       "  5814,\n",
       "  5815,\n",
       "  5816,\n",
       "  5818,\n",
       "  5819,\n",
       "  5827,\n",
       "  5828,\n",
       "  5835,\n",
       "  5838,\n",
       "  5842,\n",
       "  5847,\n",
       "  5862,\n",
       "  5878,\n",
       "  5879,\n",
       "  5880,\n",
       "  5894,\n",
       "  5895,\n",
       "  5904,\n",
       "  5910,\n",
       "  5916,\n",
       "  5918,\n",
       "  5920,\n",
       "  5927,\n",
       "  5933,\n",
       "  5937,\n",
       "  5940,\n",
       "  5952,\n",
       "  5953,\n",
       "  5954,\n",
       "  5968,\n",
       "  5970,\n",
       "  5973,\n",
       "  5978,\n",
       "  5979,\n",
       "  6001,\n",
       "  6002,\n",
       "  6003,\n",
       "  6007,\n",
       "  6010,\n",
       "  6012,\n",
       "  6022,\n",
       "  6034,\n",
       "  6038,\n",
       "  6041,\n",
       "  6042,\n",
       "  6044,\n",
       "  6051,\n",
       "  6052,\n",
       "  6054,\n",
       "  6061,\n",
       "  6063,\n",
       "  6066,\n",
       "  6072,\n",
       "  6080,\n",
       "  6081,\n",
       "  6099,\n",
       "  6100,\n",
       "  6101,\n",
       "  6102,\n",
       "  6104,\n",
       "  6112,\n",
       "  6120,\n",
       "  6121,\n",
       "  6122,\n",
       "  6127,\n",
       "  6128,\n",
       "  6131,\n",
       "  6133,\n",
       "  6136,\n",
       "  6137,\n",
       "  6145,\n",
       "  6146,\n",
       "  6154,\n",
       "  6156,\n",
       "  6157,\n",
       "  6158,\n",
       "  6160,\n",
       "  6164,\n",
       "  6170,\n",
       "  6172,\n",
       "  6175,\n",
       "  6181,\n",
       "  6182,\n",
       "  6183,\n",
       "  6184,\n",
       "  6185,\n",
       "  6186,\n",
       "  6188,\n",
       "  6189,\n",
       "  6190,\n",
       "  6197,\n",
       "  6210,\n",
       "  6218,\n",
       "  6220,\n",
       "  6221,\n",
       "  6225,\n",
       "  6231,\n",
       "  6232,\n",
       "  6233,\n",
       "  6239,\n",
       "  6242,\n",
       "  6245,\n",
       "  6246,\n",
       "  6262,\n",
       "  6264,\n",
       "  6270,\n",
       "  6271,\n",
       "  6277,\n",
       "  6278,\n",
       "  6292,\n",
       "  6296,\n",
       "  6299,\n",
       "  6304,\n",
       "  6305,\n",
       "  6310,\n",
       "  6317,\n",
       "  6322,\n",
       "  6325,\n",
       "  6327,\n",
       "  6336,\n",
       "  6355,\n",
       "  6362,\n",
       "  6374,\n",
       "  6381,\n",
       "  6387,\n",
       "  6400,\n",
       "  6401,\n",
       "  6409,\n",
       "  6429,\n",
       "  6431,\n",
       "  6433,\n",
       "  6436,\n",
       "  6447,\n",
       "  6455,\n",
       "  6456,\n",
       "  6460,\n",
       "  6466,\n",
       "  6467,\n",
       "  6472,\n",
       "  6481,\n",
       "  6483,\n",
       "  6491,\n",
       "  6495,\n",
       "  6542,\n",
       "  6560,\n",
       "  6567,\n",
       "  6568,\n",
       "  6582,\n",
       "  6587,\n",
       "  6618,\n",
       "  6619,\n",
       "  6621,\n",
       "  6622,\n",
       "  6623,\n",
       "  6624,\n",
       "  6625,\n",
       "  6627,\n",
       "  6628,\n",
       "  6630,\n",
       "  6631,\n",
       "  6632,\n",
       "  6642,\n",
       "  6645,\n",
       "  6656,\n",
       "  6659,\n",
       "  6664,\n",
       "  6666,\n",
       "  6675,\n",
       "  6682,\n",
       "  6686,\n",
       "  6697,\n",
       "  6708,\n",
       "  6731,\n",
       "  6733,\n",
       "  6740,\n",
       "  6743,\n",
       "  6746,\n",
       "  6747,\n",
       "  6748,\n",
       "  6749,\n",
       "  6750,\n",
       "  6753,\n",
       "  6762,\n",
       "  6769,\n",
       "  6773,\n",
       "  6774,\n",
       "  6786,\n",
       "  6788,\n",
       "  6789,\n",
       "  6792,\n",
       "  6797,\n",
       "  6799,\n",
       "  6801,\n",
       "  6805,\n",
       "  6806,\n",
       "  6808,\n",
       "  6816,\n",
       "  6822,\n",
       "  6825,\n",
       "  6826,\n",
       "  6835,\n",
       "  6836,\n",
       "  6839,\n",
       "  6840,\n",
       "  6841,\n",
       "  6846,\n",
       "  6852,\n",
       "  6853,\n",
       "  6854,\n",
       "  6855,\n",
       "  6857,\n",
       "  6859,\n",
       "  6861,\n",
       "  6864,\n",
       "  6875,\n",
       "  6885,\n",
       "  6886,\n",
       "  6887,\n",
       "  6889,\n",
       "  6890,\n",
       "  6892,\n",
       "  6921,\n",
       "  6922,\n",
       "  6923,\n",
       "  6933,\n",
       "  6936,\n",
       "  6939,\n",
       "  6954,\n",
       "  6967,\n",
       "  6970,\n",
       "  6984,\n",
       "  6985,\n",
       "  6986,\n",
       "  6996,\n",
       "  6997,\n",
       "  7009,\n",
       "  7019,\n",
       "  7023,\n",
       "  7024,\n",
       "  7026,\n",
       "  7028,\n",
       "  7043,\n",
       "  7049,\n",
       "  7059,\n",
       "  7070,\n",
       "  7072,\n",
       "  7073,\n",
       "  7080,\n",
       "  7101,\n",
       "  7102,\n",
       "  7104,\n",
       "  7106,\n",
       "  7107,\n",
       "  7114,\n",
       "  7121,\n",
       "  7129,\n",
       "  7130,\n",
       "  7133,\n",
       "  7170,\n",
       "  7190,\n",
       "  7201,\n",
       "  7207,\n",
       "  7208,\n",
       "  7209,\n",
       "  7210,\n",
       "  7215,\n",
       "  7225,\n",
       "  7237,\n",
       "  7256,\n",
       "  7263,\n",
       "  7267,\n",
       "  7310,\n",
       "  7319,\n",
       "  7329,\n",
       "  7330,\n",
       "  7332,\n",
       "  7333,\n",
       "  7335,\n",
       "  7347,\n",
       "  7361,\n",
       "  7370,\n",
       "  7373,\n",
       "  7378,\n",
       "  7382,\n",
       "  7392,\n",
       "  7401,\n",
       "  7407,\n",
       "  7409,\n",
       "  7412,\n",
       "  7421,\n",
       "  7423,\n",
       "  7438,\n",
       "  7439,\n",
       "  7440,\n",
       "  7469,\n",
       "  7496,\n",
       "  7509,\n",
       "  7511,\n",
       "  7517,\n",
       "  7524,\n",
       "  7525,\n",
       "  7526,\n",
       "  7528,\n",
       "  7542,\n",
       "  7574,\n",
       "  7578,\n",
       "  7589,\n",
       "  7590,\n",
       "  7591,\n",
       "  7595,\n",
       "  7596,\n",
       "  7597,\n",
       "  7615,\n",
       "  7616,\n",
       "  7617,\n",
       "  7618,\n",
       "  7620,\n",
       "  7623,\n",
       "  7624,\n",
       "  7626,\n",
       "  7629,\n",
       "  7630,\n",
       "  7632,\n",
       "  7633,\n",
       "  7634,\n",
       "  7640,\n",
       "  7641,\n",
       "  7643,\n",
       "  7647,\n",
       "  7651,\n",
       "  7654,\n",
       "  7661,\n",
       "  7666,\n",
       "  7673,\n",
       "  7678,\n",
       "  7679,\n",
       "  7685,\n",
       "  ...]}"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subgraph_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "37670ce7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(subgraph_data[0]['features'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "9103a2c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 381,  381,  381,  ..., 1510, 1512, 1525],\n",
       "        [ 679,  224,  647,  ..., 1510, 1512, 1525]], device='cuda:0')"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subgraph_data[0]['edges'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "ad4617a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1526"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(subgraph_data[0]['global_idx'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "92a1a14d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26190"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(subgraph_data[0]['edges'][0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "640a87ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "893320"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(subgraph_data[0]['edges'][1][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "b82c173f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "147584"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(subgraph_data[0]['edges'][2][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f4179bdc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1526\n",
      "2000\n",
      "21\n",
      "993\n",
      "6455\n",
      "322\n",
      "3101\n",
      "2816\n"
     ]
    }
   ],
   "source": [
    "for sg in subgraph_data:\n",
    "    print(len(sg['global_idx']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de47b7dc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fe60b3a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         ...,\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
       "\n",
       "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         ...,\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
       "\n",
       "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         ...,\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
       "\n",
       "        ...,\n",
       "\n",
       "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         ...,\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
       "\n",
       "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         ...,\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan]],\n",
       "\n",
       "        [[nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         ...,\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan],\n",
       "         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:0')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings_u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f0ba806",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c0a5b45",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "fccf6a58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 51, 2])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out.shape\n",
    "x_softmax = torch.exp(logits).cpu().detach()\n",
    "positive_class_probs = x_softmax[:, 1].numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "86f8eb94",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_softmax = torch.exp(out).cpu().detach()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8c604b25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 51, 2])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_softmax.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5bc0e096",
   "metadata": {},
   "outputs": [],
   "source": [
    "positive_class_probs = x_softmax[:, 0, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0aef89cc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "positive_class_probs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "222c7cd4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f4e96ad",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e85868dc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2113ffb7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ce6a76f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a95c6161",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4e3cf91",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0956d2c0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6434cdc8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "203488ea",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a104777e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5be66e49",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "242a0ff1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1516612a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1722569e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35f002f1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tsne01",
   "language": "python",
   "name": "tsne01"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
